import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, GRU, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/LSTM_3d_TFM'
TIME_STEPS=432 #3d
CMODEL = LSTM
UNITS=40
DROPOUT=0.405
ACTIVATION='tanh'
OPTIMIZER='adam'
EPOCHS=68
BATCHSIZE=45
VALIDATIONSPLIT=0.1
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (2685, 432, 1) y_train shape: (2685,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(CMODEL(units = UNITS, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Dropout(rate=DROPOUT))
model.add(TimeDistributed(Dense(1,kernel_initializer='normal',activation=ACTIVATION)))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=['mse',rmse])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= lstm (LSTM) (None, 432, 40) 6720 _________________________________________________________________ dropout (Dropout) (None, 432, 40) 0 _________________________________________________________________ time_distributed (TimeDistri (None, 432, 1) 41 ================================================================= Total params: 6,761 Trainable params: 6,761 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/68 54/54 [==============================] - 8s 144ms/step - loss: 0.7806 - mse: 0.9856 - rmse: 0.7826 - val_loss: 0.8897 - val_mse: 0.8320 - val_rmse: 0.8898 Epoch 2/68 54/54 [==============================] - 8s 157ms/step - loss: 0.7665 - mse: 0.9582 - rmse: 0.7678 - val_loss: 0.7797 - val_mse: 0.6445 - val_rmse: 0.7809 Epoch 3/68 54/54 [==============================] - 8s 139ms/step - loss: 0.7627 - mse: 0.9600 - rmse: 0.7697 - val_loss: 0.7143 - val_mse: 0.5454 - val_rmse: 0.7170 Epoch 4/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7579 - mse: 0.9596 - rmse: 0.7681 - val_loss: 0.6640 - val_mse: 0.4742 - val_rmse: 0.6677 Epoch 5/68 54/54 [==============================] - 7s 136ms/step - loss: 0.7554 - mse: 0.9629 - rmse: 0.7680 - val_loss: 0.6285 - val_mse: 0.4270 - val_rmse: 0.6330 Epoch 6/68 54/54 [==============================] - 9s 163ms/step - loss: 0.7541 - mse: 0.9668 - rmse: 0.7682 - val_loss: 0.6037 - val_mse: 0.3956 - val_rmse: 0.6089 Epoch 7/68 54/54 [==============================] - 8s 142ms/step - loss: 0.7529 - mse: 0.9703 - rmse: 0.7679 - val_loss: 0.5872 - val_mse: 0.3754 - val_rmse: 0.5929 Epoch 8/68 54/54 [==============================] - 7s 136ms/step - loss: 0.7511 - mse: 0.9714 - rmse: 0.7671 - val_loss: 0.5733 - val_mse: 0.3588 - val_rmse: 0.5794 Epoch 9/68 54/54 [==============================] - 7s 129ms/step - loss: 0.7497 - mse: 0.9729 - rmse: 0.7664 - val_loss: 0.5602 - val_mse: 0.3436 - val_rmse: 0.5668 Epoch 10/68 54/54 [==============================] - 7s 135ms/step - loss: 0.7487 - mse: 0.9750 - rmse: 0.7662 - val_loss: 0.5506 - val_mse: 0.3327 - val_rmse: 0.5575 Epoch 11/68 54/54 [==============================] - 7s 138ms/step - loss: 0.7480 - mse: 0.9767 - rmse: 0.7659 - val_loss: 0.5426 - val_mse: 0.3237 - val_rmse: 0.5498 Epoch 12/68 54/54 [==============================] - 7s 135ms/step - loss: 0.7475 - mse: 0.9783 - rmse: 0.7658 - val_loss: 0.5366 - val_mse: 0.3171 - val_rmse: 0.5440 Epoch 13/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7470 - mse: 0.9792 - rmse: 0.7655 - val_loss: 0.5315 - val_mse: 0.3116 - val_rmse: 0.5391 Epoch 14/68 54/54 [==============================] - 7s 137ms/step - loss: 0.7466 - mse: 0.9804 - rmse: 0.7655 - val_loss: 0.5276 - val_mse: 0.3073 - val_rmse: 0.5354 Epoch 15/68 54/54 [==============================] - 7s 133ms/step - loss: 0.7461 - mse: 0.9806 - rmse: 0.7650 - val_loss: 0.5242 - val_mse: 0.3037 - val_rmse: 0.5322 Epoch 16/68 54/54 [==============================] - 7s 132ms/step - loss: 0.7458 - mse: 0.9813 - rmse: 0.7649 - val_loss: 0.5210 - val_mse: 0.3003 - val_rmse: 0.5291 Epoch 17/68 54/54 [==============================] - 7s 137ms/step - loss: 0.7454 - mse: 0.9820 - rmse: 0.7648 - val_loss: 0.5184 - val_mse: 0.2977 - val_rmse: 0.5268 Epoch 18/68 54/54 [==============================] - 7s 135ms/step - loss: 0.7451 - mse: 0.9823 - rmse: 0.7646 - val_loss: 0.5159 - val_mse: 0.2950 - val_rmse: 0.5244 Epoch 19/68 54/54 [==============================] - 8s 140ms/step - loss: 0.7447 - mse: 0.9826 - rmse: 0.7644 - val_loss: 0.5136 - val_mse: 0.2926 - val_rmse: 0.5222 Epoch 20/68 54/54 [==============================] - 7s 133ms/step - loss: 0.7445 - mse: 0.9828 - rmse: 0.7643 - val_loss: 0.5120 - val_mse: 0.2911 - val_rmse: 0.5208 Epoch 21/68 54/54 [==============================] - 7s 136ms/step - loss: 0.7445 - mse: 0.9833 - rmse: 0.7644 - val_loss: 0.5105 - val_mse: 0.2896 - val_rmse: 0.5195 Epoch 22/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7442 - mse: 0.9833 - rmse: 0.7642 - val_loss: 0.5092 - val_mse: 0.2883 - val_rmse: 0.5183 Epoch 23/68 54/54 [==============================] - 7s 126ms/step - loss: 0.7439 - mse: 0.9833 - rmse: 0.7640 - val_loss: 0.5074 - val_mse: 0.2865 - val_rmse: 0.5167 Epoch 24/68 54/54 [==============================] - 7s 137ms/step - loss: 0.7437 - mse: 0.9835 - rmse: 0.7640 - val_loss: 0.5059 - val_mse: 0.2850 - val_rmse: 0.5153 Epoch 25/68 54/54 [==============================] - 7s 136ms/step - loss: 0.7434 - mse: 0.9837 - rmse: 0.7638 - val_loss: 0.5045 - val_mse: 0.2836 - val_rmse: 0.5141 Epoch 26/68 54/54 [==============================] - 7s 137ms/step - loss: 0.7433 - mse: 0.9838 - rmse: 0.7638 - val_loss: 0.5036 - val_mse: 0.2827 - val_rmse: 0.5133 Epoch 27/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7432 - mse: 0.9838 - rmse: 0.7638 - val_loss: 0.5026 - val_mse: 0.2818 - val_rmse: 0.5125 Epoch 28/68 54/54 [==============================] - 8s 140ms/step - loss: 0.7430 - mse: 0.9838 - rmse: 0.7636 - val_loss: 0.5014 - val_mse: 0.2807 - val_rmse: 0.5115 Epoch 29/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7427 - mse: 0.9838 - rmse: 0.7635 - val_loss: 0.5003 - val_mse: 0.2796 - val_rmse: 0.5105 Epoch 30/68 54/54 [==============================] - 8s 140ms/step - loss: 0.7427 - mse: 0.9838 - rmse: 0.7636 - val_loss: 0.4994 - val_mse: 0.2787 - val_rmse: 0.5097 Epoch 31/68 54/54 [==============================] - 8s 140ms/step - loss: 0.7423 - mse: 0.9836 - rmse: 0.7633 - val_loss: 0.4984 - val_mse: 0.2778 - val_rmse: 0.5088 Epoch 32/68 54/54 [==============================] - 8s 145ms/step - loss: 0.7424 - mse: 0.9840 - rmse: 0.7635 - val_loss: 0.4974 - val_mse: 0.2769 - val_rmse: 0.5080 Epoch 33/68 54/54 [==============================] - 7s 136ms/step - loss: 0.7422 - mse: 0.9839 - rmse: 0.7634 - val_loss: 0.4967 - val_mse: 0.2762 - val_rmse: 0.5074 Epoch 34/68 54/54 [==============================] - 7s 133ms/step - loss: 0.7421 - mse: 0.9838 - rmse: 0.7633 - val_loss: 0.4959 - val_mse: 0.2755 - val_rmse: 0.5068 Epoch 35/68 54/54 [==============================] - 8s 140ms/step - loss: 0.7420 - mse: 0.9841 - rmse: 0.7633 - val_loss: 0.4949 - val_mse: 0.2746 - val_rmse: 0.5060 Epoch 36/68 54/54 [==============================] - 7s 131ms/step - loss: 0.7418 - mse: 0.9838 - rmse: 0.7632 - val_loss: 0.4938 - val_mse: 0.2735 - val_rmse: 0.5049 Epoch 37/68 54/54 [==============================] - 8s 144ms/step - loss: 0.7417 - mse: 0.9840 - rmse: 0.7633 - val_loss: 0.4937 - val_mse: 0.2735 - val_rmse: 0.5050 Epoch 38/68 54/54 [==============================] - 7s 138ms/step - loss: 0.7417 - mse: 0.9839 - rmse: 0.7633 - val_loss: 0.4933 - val_mse: 0.2732 - val_rmse: 0.5047 Epoch 39/68 54/54 [==============================] - 7s 135ms/step - loss: 0.7414 - mse: 0.9837 - rmse: 0.7632 - val_loss: 0.4925 - val_mse: 0.2725 - val_rmse: 0.5041 Epoch 40/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7413 - mse: 0.9835 - rmse: 0.7631 - val_loss: 0.4917 - val_mse: 0.2718 - val_rmse: 0.5034 Epoch 41/68 54/54 [==============================] - 7s 139ms/step - loss: 0.7412 - mse: 0.9836 - rmse: 0.7631 - val_loss: 0.4911 - val_mse: 0.2712 - val_rmse: 0.5030 Epoch 42/68 54/54 [==============================] - 8s 141ms/step - loss: 0.7411 - mse: 0.9837 - rmse: 0.7632 - val_loss: 0.4905 - val_mse: 0.2707 - val_rmse: 0.5025 Epoch 43/68 54/54 [==============================] - 8s 140ms/step - loss: 0.7409 - mse: 0.9834 - rmse: 0.7631 - val_loss: 0.4897 - val_mse: 0.2700 - val_rmse: 0.5019 Epoch 44/68 54/54 [==============================] - 7s 132ms/step - loss: 0.7408 - mse: 0.9833 - rmse: 0.7631 - val_loss: 0.4888 - val_mse: 0.2692 - val_rmse: 0.5011 Epoch 45/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7408 - mse: 0.9834 - rmse: 0.7632 - val_loss: 0.4882 - val_mse: 0.2687 - val_rmse: 0.5006 Epoch 46/68 54/54 [==============================] - 8s 141ms/step - loss: 0.7406 - mse: 0.9833 - rmse: 0.7632 - val_loss: 0.4876 - val_mse: 0.2682 - val_rmse: 0.5002 Epoch 47/68 54/54 [==============================] - 7s 136ms/step - loss: 0.7404 - mse: 0.9830 - rmse: 0.7631 - val_loss: 0.4868 - val_mse: 0.2675 - val_rmse: 0.4995 Epoch 48/68 54/54 [==============================] - 8s 139ms/step - loss: 0.7404 - mse: 0.9832 - rmse: 0.7632 - val_loss: 0.4862 - val_mse: 0.2670 - val_rmse: 0.4991 Epoch 49/68 54/54 [==============================] - 8s 146ms/step - loss: 0.7403 - mse: 0.9830 - rmse: 0.7631 - val_loss: 0.4853 - val_mse: 0.2662 - val_rmse: 0.4983 Epoch 50/68 54/54 [==============================] - 8s 140ms/step - loss: 0.7402 - mse: 0.9830 - rmse: 0.7632 - val_loss: 0.4850 - val_mse: 0.2660 - val_rmse: 0.4982 Epoch 51/68 54/54 [==============================] - 7s 131ms/step - loss: 0.7403 - mse: 0.9833 - rmse: 0.7633 - val_loss: 0.4846 - val_mse: 0.2656 - val_rmse: 0.4979 Epoch 52/68 54/54 [==============================] - 7s 131ms/step - loss: 0.7400 - mse: 0.9830 - rmse: 0.7631 - val_loss: 0.4841 - val_mse: 0.2652 - val_rmse: 0.4974 Epoch 53/68 54/54 [==============================] - 7s 134ms/step - loss: 0.7400 - mse: 0.9830 - rmse: 0.7633 - val_loss: 0.4832 - val_mse: 0.2644 - val_rmse: 0.4968 Epoch 54/68 54/54 [==============================] - 8s 144ms/step - loss: 0.7399 - mse: 0.9830 - rmse: 0.7633 - val_loss: 0.4828 - val_mse: 0.2641 - val_rmse: 0.4964 Epoch 55/68 54/54 [==============================] - 8s 152ms/step - loss: 0.7398 - mse: 0.9830 - rmse: 0.7633 - val_loss: 0.4818 - val_mse: 0.2632 - val_rmse: 0.4956 Epoch 56/68 54/54 [==============================] - 7s 137ms/step - loss: 0.7398 - mse: 0.9833 - rmse: 0.7634 - val_loss: 0.4815 - val_mse: 0.2630 - val_rmse: 0.4954 Epoch 57/68 54/54 [==============================] - 7s 138ms/step - loss: 0.7395 - mse: 0.9828 - rmse: 0.7632 - val_loss: 0.4806 - val_mse: 0.2621 - val_rmse: 0.4945 Epoch 58/68 54/54 [==============================] - 8s 146ms/step - loss: 0.7396 - mse: 0.9835 - rmse: 0.7635 - val_loss: 0.4801 - val_mse: 0.2617 - val_rmse: 0.4942 Epoch 59/68 54/54 [==============================] - 8s 151ms/step - loss: 0.7393 - mse: 0.9832 - rmse: 0.7633 - val_loss: 0.4793 - val_mse: 0.2608 - val_rmse: 0.4933 Epoch 60/68 54/54 [==============================] - 8s 154ms/step - loss: 0.7397 - mse: 0.9844 - rmse: 0.7639 - val_loss: 0.4785 - val_mse: 0.2602 - val_rmse: 0.4928 Epoch 61/68 54/54 [==============================] - 8s 156ms/step - loss: 0.7390 - mse: 0.9836 - rmse: 0.7633 - val_loss: 0.4778 - val_mse: 0.2594 - val_rmse: 0.4919 Epoch 62/68 54/54 [==============================] - 8s 152ms/step - loss: 0.7388 - mse: 0.9842 - rmse: 0.7636 - val_loss: 0.4764 - val_mse: 0.2582 - val_rmse: 0.4907 Epoch 63/68 54/54 [==============================] - 8s 142ms/step - loss: 0.7381 - mse: 0.9839 - rmse: 0.7633 - val_loss: 0.4747 - val_mse: 0.2568 - val_rmse: 0.4893 Epoch 64/68 54/54 [==============================] - 8s 139ms/step - loss: 0.7481 - mse: 1.0051 - rmse: 0.7732 - val_loss: 0.4846 - val_mse: 0.2663 - val_rmse: 0.4987 Epoch 65/68 54/54 [==============================] - 8s 150ms/step - loss: 0.7415 - mse: 0.9913 - rmse: 0.7636 - val_loss: 0.4843 - val_mse: 0.2661 - val_rmse: 0.4985 Epoch 66/68 54/54 [==============================] - 8s 142ms/step - loss: 0.7382 - mse: 0.9816 - rmse: 0.7615 - val_loss: 0.4801 - val_mse: 0.2620 - val_rmse: 0.4946 Epoch 67/68 54/54 [==============================] - 7s 137ms/step - loss: 0.7377 - mse: 0.9813 - rmse: 0.7618 - val_loss: 0.4768 - val_mse: 0.2589 - val_rmse: 0.4916 Epoch 68/68 54/54 [==============================] - 7s 137ms/step - loss: 0.7373 - mse: 0.9813 - rmse: 0.7622 - val_loss: 0.4738 - val_mse: 0.2560 - val_rmse: 0.4888
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,"LSTM")
LSTM: Mean Absolute Error: 0.6263 Root Mean Square Error: 0.9317 Mean Square Error: 0.8680
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.80 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,"LSTM")
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ########################### Testing shape: (348, 432, 1) 3/11 [=======>......................] - ETA: 0s - loss: 0.4703 - mse: 0.4651 - rmse: 0.5047
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
11/11 [==============================] - 0s 29ms/step - loss: 0.6872 - mse: 1.1191 - rmse: 0.7162 evaluate: [0.6872174739837646, 1.119050145149231, 0.716234564781189] LSTM: Mean Absolute Error: 0.5033 Root Mean Square Error: 0.8603 Mean Square Error: 0.7401
anomalies: (348, 10)
###################################################### ####################### PM25 ########################### Testing shape: (348, 432, 1) 3/11 [=======>......................] - ETA: 0s - loss: 0.5009 - mse: 0.5162 - rmse: 0.53
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
11/11 [==============================] - 0s 24ms/step - loss: 0.7329 - mse: 1.2526 - rmse: 0.7622 evaluate: [0.7328898310661316, 1.252611517906189, 0.7622219324111938] LSTM: Mean Absolute Error: 0.5372 Root Mean Square Error: 0.8776 Mean Square Error: 0.7703
anomalies: (348, 10)
###################################################### ####################### PM10 ########################### Testing shape: (348, 432, 1) 5/11 [============>.................] - ETA: 0s - loss: 0.4564 - mse: 0.3978 - rmse: 0.4945
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
11/11 [==============================] - 0s 27ms/step - loss: 0.7622 - mse: 1.3354 - rmse: 0.7928 evaluate: [0.7622094750404358, 1.3354195356369019, 0.7927601337432861] LSTM: Mean Absolute Error: 0.5608 Root Mean Square Error: 0.8876 Mean Square Error: 0.7878
anomalies: (161, 10)
###################################################### ####################### PM1ATM ########################### Testing shape: (348, 432, 1) 3/11 [=======>......................] - ETA: 0s - loss: 0.5378 - mse: 0.5469 - rmse: 0.57
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
11/11 [==============================] - 0s 24ms/step - loss: 0.7605 - mse: 1.1737 - rmse: 0.7905 evaluate: [0.7605029344558716, 1.1736979484558105, 0.790486752986908] LSTM: Mean Absolute Error: 0.5647 Root Mean Square Error: 0.8704 Mean Square Error: 0.7577
anomalies: (117, 10)
###################################################### ####################### PM25ATM ########################### Testing shape: (348, 432, 1) 5/11 [============>.................] - ETA: 0s - loss: 0.4714 - mse: 0.3967 - rmse: 0.5086
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
11/11 [==============================] - 0s 25ms/step - loss: 0.7498 - mse: 1.1573 - rmse: 0.7796 evaluate: [0.7497873306274414, 1.1572949886322021, 0.7796124815940857] LSTM: Mean Absolute Error: 0.5552 Root Mean Square Error: 0.8631 Mean Square Error: 0.7450
anomalies: (200, 10)
###################################################### ####################### PM10ATM ########################### Testing shape: (348, 432, 1) 3/11 [=======>......................] - ETA: 0s - loss: 0.5140 - mse: 0.5307 - rmse: 0.5507
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
11/11 [==============================] - 0s 24ms/step - loss: 0.7457 - mse: 1.1805 - rmse: 0.7761 evaluate: [0.7456705570220947, 1.1805392503738403, 0.7761043906211853] LSTM: Mean Absolute Error: 0.5553 Root Mean Square Error: 0.8714 Mean Square Error: 0.7593
anomalies: (82, 10)
######################################################